// -*- c -*-
//
// %CopyrightBegin%
//
// SPDX-License-Identifier: Apache-2.0
//
// Copyright Ericsson AB 2017-2025. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// %CopyrightEnd%
//

//
// Stack manipulation instructions follow.
//
// See the comment for AH() in macros.tab for information about
// the layout of stack frames.
//

allocate(NeedStack, Live) {
    $AH($NeedStack, 0, $Live);
}

allocate_heap(NeedStack, NeedHeap, Live) {
    $AH($NeedStack, $NeedHeap, $Live);
}

allocate_init(NeedStack, Live, Y) {
    $AH($NeedStack, 0, $Live);
    make_blank($Y);
}

i_allocate_zero(NeedStack, Live) {
    Eterm* ptr;
    int i = $NeedStack;
    $AH(i, 0, $Live);
    for (ptr = E + i; ptr > E; ptr--) {
	 make_blank(*ptr);
    }
}

i_allocate_heap_zero(NeedStack, NeedHeap, Live) {
    Eterm* ptr;
    int i = $NeedStack;
    $AH(i, $NeedHeap, $Live);
    for (ptr = E + i; ptr > E; ptr--) {
        make_blank(*ptr);
    }
}

// This instruction is probably never used (because it is combined with a
// a return). However, a future compiler might for some reason emit a
// deallocate not followed by a return, and that should work.

deallocate(Deallocate) {
    //| -no_prefetch
    E = ADD_BYTE_OFFSET(E, $Deallocate);
}

//
// Micro-benchmarks showed that the deallocate_return instruction
// became slower when the continuation pointer was moved from
// the process struct to the stack. The reason seems to be read
// dependencies, i.e. that the CPU cannot figure out beforehand
// from which position on the stack the continuation pointer
// should be fetched.
//
// Initializing num_bytes with a constant value seems to restore
// the lost speed, so we've specialized the instruction for the
// most common values.
//

deallocate_return0 := dealloc_ret.n0.execute;
deallocate_return1 := dealloc_ret.n1.execute;
deallocate_return2 := dealloc_ret.n2.execute;
deallocate_return3 := dealloc_ret.n3.execute;
deallocate_return4 := dealloc_ret.n4.execute;
deallocate_return := dealloc_ret.var.execute;

dealloc_ret.head() {
    Uint num_bytes;
}

dealloc_ret.n0() {
    num_bytes = (0+1) * sizeof(Eterm);
}

dealloc_ret.n1() {
    num_bytes = (1+1) * sizeof(Eterm);
}

dealloc_ret.n2() {
    num_bytes = (2+1) * sizeof(Eterm);
}

dealloc_ret.n3() {
    num_bytes = (3+1) * sizeof(Eterm);
}

dealloc_ret.n4() {
    num_bytes = (4+1) * sizeof(Eterm);
}

dealloc_ret.var(Deallocate) {
    num_bytes = $Deallocate;
}

dealloc_ret.execute() {
    //| -no_next

    E = ADD_BYTE_OFFSET(E, num_bytes);
    $RETURN();
    CHECK_TERM(x(0));
    $DISPATCH_RETURN();
}

move_deallocate_return(Src, Deallocate) {
    //| -no_next

    /*
     * Explicitly do reads first to mitigate the impact of read
     * dependencies.
     */

    Uint bytes_to_pop = $Deallocate;
    Eterm src = $Src;
    E = ADD_BYTE_OFFSET(E, bytes_to_pop);
    x(0) = src;
    DTRACE_RETURN_FROM_PC(c_p, I);
    $RETURN();
    CHECK_TERM(x(0));
    $DISPATCH_RETURN();
}

// Call instructions

i_call(CallDest) {
    //| -no_next
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    $DISPATCH_REL($CallDest);
}

move_call(Src, CallDest) {
    //| -no_next
    Eterm call_dest = $CallDest;
    Eterm src = $Src;
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    x(0) = src;
    $DISPATCH_REL(call_dest);
}

i_call_last(CallDest, Deallocate) {
    //| -no_next
    $deallocate($Deallocate);
    $DISPATCH_REL($CallDest);
}

move_call_last(Src, CallDest, Deallocate) {
    //| -no_next
    Eterm call_dest = $CallDest;
    Eterm src = $Src;
    $deallocate($Deallocate);
    x(0) = src;
    $DISPATCH_REL(call_dest);
}

i_call_only(CallDest) {
    //| -no_next
    $DISPATCH_REL($CallDest);
}

move_call_only(Src, CallDest) {
    //| -no_next
    Eterm call_dest = $CallDest;
    Eterm src = $Src;
    x(0) = src;
    $DISPATCH_REL(call_dest);
}

i_call_ext(Dest) {
    //| -no_next
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    $DISPATCH_EXPORT($Dest);
}

i_move_call_ext(Src, CallDest) {
    //| -no_next
    Eterm call_dest = $CallDest;
    Eterm src = $Src;
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    x(0) = src;
    $DISPATCH_EXPORT(call_dest);
}

i_call_ext_only(Dest) {
    //| -no_next
    $DISPATCH_EXPORT($Dest);
}

i_move_call_ext_only(CallDest, Src) {
    //| -no_next
    Eterm call_dest = $CallDest;
    Eterm src = $Src;
    x(0) = src;
    $DISPATCH_EXPORT(call_dest);
}

i_call_ext_last(Dest, Deallocate) {
    //| -no_next
    $deallocate($Deallocate);
    $DISPATCH_EXPORT($Dest);
}

i_move_call_ext_last(CallDest, Deallocate, Src) {
    //| -no_next
    Eterm call_dest = $CallDest;
    Eterm src = $Src;
    $deallocate($Deallocate);
    x(0) = src;
    $DISPATCH_EXPORT(call_dest);
}

APPLY(I, Deallocate, Next) {
    //| -no_next
    const Export *ep;

    HEAVY_SWAPOUT;
    ep = apply(c_p, reg, $I, $Deallocate);
    HEAVY_SWAPIN;

    if (ERTS_UNLIKELY(ep == NULL)) {
        $HANDLE_APPLY_ERROR();
    } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
        save_calls(c_p, ep);
    }

    $Next = ep->dispatch.addresses[erts_active_code_ix()];
}

HANDLE_APPLY_ERROR() {
    static ErtsCodeMFA apply3_mfa = {am_erlang, am_apply, 3};
    I = handle_error(c_p, I, reg, &apply3_mfa);
    goto post_error_handling;
}

i_apply() {
    //| -no_next
    const BeamInstr *next;
    $APPLY(NULL, 0, next);

    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    $DISPATCH_ABS(next);
}

i_apply_last(Deallocate) {
    //| -no_next
    const BeamInstr *next;
    $APPLY(I, $Deallocate, next);
    $deallocate($Deallocate);
    $DISPATCH_ABS(next);
}

i_apply_only() {
    //| -no_next
    const BeamInstr *next;
    $APPLY(I, 0, next);
    $DISPATCH_ABS(next);
}

FIXED_APPLY(Arity, I, Deallocate, Next) {
    //| -no_next
    const Export *ep;

    HEAVY_SWAPOUT;
    ep = fixed_apply(c_p, reg, $Arity, $I, $Deallocate);
    HEAVY_SWAPIN;

    if (ERTS_UNLIKELY(ep == NULL)) {
        $HANDLE_APPLY_ERROR();
    } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
        save_calls(c_p, ep);
    }

    $Next = ep->dispatch.addresses[erts_active_code_ix()];
}

apply(Arity) {
    //| -no_next
    const BeamInstr *next;
    $FIXED_APPLY($Arity, NULL, 0, next);
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    $DISPATCH_ABS(next);
}

apply_last(Arity, Deallocate) {
    //| -no_next
    const BeamInstr *next;
    $FIXED_APPLY($Arity, I, $Deallocate, next);
    $deallocate($Deallocate);
    $DISPATCH_ABS(next);
}

APPLY_FUN(Next) {
    HEAVY_SWAPOUT;
    $Next = apply_fun(c_p, x(0), x(1), reg);
    HEAVY_SWAPIN;

    if (ERTS_UNLIKELY(next == NULL)) {
        $HANDLE_APPLY_FUN_ERROR();
    }
}

HANDLE_APPLY_FUN_ERROR() {
     goto find_func_info;
}

i_apply_fun() {
    //| -no_next
    const BeamInstr *next;
    $APPLY_FUN(next);
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    $DISPATCH_FUN(next);
}

i_apply_fun_last(Deallocate) {
    //| -no_next
    const BeamInstr *next;
    $APPLY_FUN(next);
    $deallocate($Deallocate);
    $DISPATCH_FUN(next);
}

i_apply_fun_only() {
    //| -no_next
    const BeamInstr *next;
    $APPLY_FUN(next);
    $DISPATCH_FUN(next);
}

CALL_FUN(Fun, Next) {
    //| -no_next

    HEAVY_SWAPOUT;
    $Next = call_fun(c_p, $Fun, reg, THE_NON_VALUE);
    HEAVY_SWAPIN;

    if (ERTS_UNLIKELY(next == NULL)) {
        $HANDLE_APPLY_FUN_ERROR();
    }
}

i_call_fun(Fun) {
    //| -no_next
    const BeamInstr *next;
    $CALL_FUN($Fun, next);
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);
    $DISPATCH_FUN(next);
}

i_call_fun_last(Fun, Deallocate) {
    //| -no_next
    const BeamInstr *next;
    $CALL_FUN($Fun, next);
    $deallocate($Deallocate);
    $DISPATCH_FUN(next);
}

return() {
    //| -no_next
    DTRACE_RETURN_FROM_PC(c_p, I);
    $RETURN();
    CHECK_TERM(x(0));
    HEAP_SPACE_VERIFIED(0);

    $DISPATCH_RETURN();
}

get_list(Src, Hd, Tl) {
    Eterm* tmp_ptr = list_val($Src);
    Eterm hd, tl;
    hd = CAR(tmp_ptr);
    tl = CDR(tmp_ptr);
    $Hd = hd;
    $Tl = tl;
}

get_hd(Src, Hd) {
    Eterm* tmp_ptr = list_val($Src);
    $Hd = CAR(tmp_ptr);
}

get_tl(Src, Tl) {
    Eterm* tmp_ptr = list_val($Src);
    $Tl = CDR(tmp_ptr);
}

i_get(Src, Dst) {
    $Dst = erts_pd_hash_get(c_p, $Src);
}

i_get_hash(Src, Hash, Dst) {
    $Dst = erts_pd_hash_get_with_hx(c_p, $Hash, $Src);
}

i_get_tuple_element(Src, Element, Dst) {
    Eterm* src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
    $Dst = *src;
}

i_get_tuple_element2(Src, Element, Dst) {
    Eterm* src;
    Eterm* dst;
    Eterm E1, E2;
    src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
    dst = &($Dst);
    E1 = src[0];
    E2 = src[1];
    dst[0] = E1;
    dst[1] = E2;
}

i_get_tuple_element2_dst(Src, Element, D1, D2) {
    Eterm* src;
    Eterm E1, E2;
    src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
    E1 = src[0];
    E2 = src[1];
    $D1 = E1;
    $D2 = E2;
}

i_get_tuple_element3(Src, Element, Dst) {
    Eterm* src;
    Eterm* dst;
    Eterm E1, E2, E3;
    src = ADD_BYTE_OFFSET(tuple_val($Src), $Element);
    dst = &($Dst);
    E1 = src[0];
    E2 = src[1];
    E3 = src[2];
    dst[0] = E1;
    dst[1] = E2;
    dst[2] = E3;
}

i_element := element_group.fetch.execute;


element_group.head() {
    Eterm element_tuple;
}

element_group.fetch(Src) {
    element_tuple = $Src;
}

element_group.execute(Fail, Index, Dst) {
    Eterm element_index = $Index;
    if (ERTS_LIKELY(is_small(element_index) && is_tuple(element_tuple))) {
        Eterm* tp = tuple_val(element_tuple);

        if ((signed_val(element_index) >= 1) &&
            (signed_val(element_index) <= arityval(*tp))) {
            $Dst = tp[signed_val(element_index)];
            $NEXT0();
        }
    }
    c_p->freason = BADARG;
    $BIF_ERROR_ARITY_2($Fail, BIF_element_2, element_index, element_tuple);
}

i_fast_element := fast_element_group.fetch.execute;

fast_element_group.head() {
    Eterm fast_element_tuple;
}

fast_element_group.fetch(Src) {
    fast_element_tuple = $Src;
}

fast_element_group.execute(Fail, Index, Dst) {
    if (ERTS_LIKELY(is_tuple(fast_element_tuple))) {
        Eterm* tp = tuple_val(fast_element_tuple);
        Eterm pos = $Index;	/* Untagged integer >= 1 */
        if (pos <= arityval(*tp)) {
            $Dst = tp[pos];
            $NEXT0();
	}
    }
    c_p->freason = BADARG;
    $BIF_ERROR_ARITY_2($Fail, BIF_element_2, make_small($Index), fast_element_tuple);
}

i_init(Y) {
    make_blank($Y);
}

i_init_seq3(Y1) {
    Eterm* dst = &$Y1;
    make_blank(dst[0]);
    make_blank(dst[1]);
    make_blank(dst[2]);
}

i_init_seq4(Y1) {
    Eterm* dst = &$Y1;
    make_blank(dst[0]);
    make_blank(dst[1]);
    make_blank(dst[2]);
    make_blank(dst[3]);
}

i_init_seq5(Y1) {
    Eterm* dst = &$Y1;
    make_blank(dst[0]);
    make_blank(dst[1]);
    make_blank(dst[2]);
    make_blank(dst[3]);
    make_blank(dst[4]);
}

i_init2(Y1, Y2) {
    make_blank($Y1);
    make_blank($Y2);
}

i_init3(Y1, Y2, Y3) {
    make_blank($Y1);
    make_blank($Y2);
    make_blank($Y3);
}

i_make_fun3(FunP, Dst, Arity, NumFree) {
    ErlFunEntry *fe = (ErlFunEntry *) $FunP;
    int i, num_free = $NumFree;
    ErlFunThing* funp;
    //| -no_next

    funp = (ErlFunThing*)HTOP;
    HTOP += ERL_FUN_SIZE + num_free;

    funp->thing_word = MAKE_FUN_HEADER($Arity, num_free, 0);
    funp->entry.fun = fe;

#ifdef DEBUG
    {
        /* Note that `mfa` may be NULL if the fun is currently being purged. We
         * ignore this since it's not an error and we only need `mfa` to
         * sanity-check the arity at this point. If the fun is called while in
         * this state, the `error_handler` module will take care of it. */
        const ErtsCodeMFA *mfa = erts_get_fun_mfa(fe, erts_active_code_ix());
        ASSERT(!mfa || fun_arity(funp) == mfa->arity - num_free);
        ASSERT($Arity == fe->arity);
    }
#endif

    I = $NEXT_INSTRUCTION;

    for (i = 0; i < num_free; i++) {
        Eterm term = *I++;
        switch (loader_tag(term)) {
        case LOADER_X_REG:
            term = x(loader_x_reg_index(term));
            break;
        case LOADER_Y_REG:
            term = y(loader_y_reg_index(term));
            break;
        }
        funp->env[i] = term;
    }

    $Dst = make_fun(funp);
    Goto(*I);
}

move_trim(Src, Dst, Words) {
    $Dst = $Src;
    $i_trim($Words);
}

i_trim(Words) {
    E += $Words;

    /*
     * Clear the reserved location for the continuation pointer at
     * E[0]. This is not strictly necessary for correctness, but if a
     * GC is triggered before E[0] is overwritten by another
     * continuation pointer the now dead term at E[0] would be
     * retained by the GC.
     */
    E[0] = NIL;
}

move(Src, Dst) {
    $Dst = $Src;
}

move3(S1, D1, S2, D2, S3, D3) {
    $D1 = $S1;
    $D2 = $S2;
    $D3 = $S3;
}

move2_par(S1, D1, S2, D2) {
    Eterm V1, V2;
    V1 = $S1;
    V2 = $S2;
    $D1 = V1;
    $D2 = V2;
}

move_shift(Src, SD, D) {
    Eterm V;
    V = $Src;
    $D = $SD;
    $SD = V;
}

move_src_window2(Src, D1, D2) {
    Eterm* src = &$Src;
    Eterm s1, s2;
    s1 = src[0];
    s2 = src[1];
    $D1 = s1;
    $D2 = s2;
}

move_src_window3(Src, D1, D2, D3) {
    Eterm* src = &$Src;
    Eterm s1, s2, s3;
    s1 = src[0];
    s2 = src[1];
    s3 = src[2];
    $D1 = s1;
    $D2 = s2;
    $D3 = s3;
}

move_src_window4(Src, D1, D2, D3, D4) {
    Eterm* src = &$Src;
    Eterm s1, s2, s3, s4;
    s1 = src[0];
    s2 = src[1];
    s3 = src[2];
    s4 = src[3];
    $D1 = s1;
    $D2 = s2;
    $D3 = s3;
    $D4 = s4;
}

move_window2(S1, S2, D) {
    Eterm xt0, xt1;
    Eterm* y = &$D;
    xt0  = $S1;
    xt1  = $S2;
    y[0] = xt0;
    y[1] = xt1;
}

move_window3(S1, S2, S3, D) {
    Eterm xt0, xt1, xt2;
    Eterm* y = &$D;
    xt0  = $S1;
    xt1  = $S2;
    xt2  = $S3;
    y[0] = xt0;
    y[1] = xt1;
    y[2] = xt2;
}

move_window4(S1, S2, S3, S4, D) {
    Eterm xt0, xt1, xt2, xt3;
    Eterm* y = &$D;
    xt0  = $S1;
    xt1  = $S2;
    xt2  = $S3;
    xt3  = $S4;
    y[0] = xt0;
    y[1] = xt1;
    y[2] = xt2;
    y[3] = xt3;
}

move_window5(S1, S2, S3, S4, S5, D) {
    Eterm xt0, xt1, xt2, xt3, xt4;
    Eterm *y = &$D;
    xt0  = $S1;
    xt1  = $S2;
    xt2  = $S3;
    xt3  = $S4;
    xt4  = $S5;
    y[0] = xt0;
    y[1] = xt1;
    y[2] = xt2;
    y[3] = xt3;
    y[4] = xt4;
}

move_return(Src) {
    //| -no_next
    x(0) = $Src;
    DTRACE_RETURN_FROM_PC(c_p, I);
    $RETURN();
    $DISPATCH_RETURN();
}

move_x1(Src) {
    x(1) = $Src;
}

move_x2(Src) {
    x(2) = $Src;
}

node(Dst) {
    $Dst = erts_this_node->sysname;
}

put_list(Hd, Tl, Dst) {
    HTOP[0] = $Hd;
    HTOP[1] = $Tl;
    $Dst = make_list(HTOP);
    HTOP += 2;
}

update_list(Hd, Dst) {
    HTOP[0] = $Hd;
    HTOP[1] = $Dst;
    $Dst = make_list(HTOP);
    HTOP += 2;
}

put_tuple2(Dst, Arity) {
    Eterm* hp = HTOP;
    Eterm arity = $Arity;

    /*
     * If operands are not packed (in the 32-bit VM),
     * it is not safe to use $Dst directly after I
     * has been updated.
     */
    Eterm* dst_ptr = &($Dst);

    //| -no_next
    ASSERT(arity != 0);
    *hp++ = make_arityval(arity);
    I = $NEXT_INSTRUCTION;
    do {
        Eterm term = *I++;
        switch (loader_tag(term)) {
        case LOADER_X_REG:
            *hp++ = x(loader_x_reg_index(term));
            break;
        case LOADER_Y_REG:
            *hp++ = y(loader_y_reg_index(term));
            break;
        default:
            *hp++ = term;
            break;
        }
    } while (--arity != 0);
    *dst_ptr = make_tuple(HTOP);
    HTOP = hp;
    ASSERT(VALID_INSTR(* (Eterm *)I));
    Goto(*I);
}

self(Dst) {
    $Dst = c_p->common.id;
}

i_update_record_copy(Size, Src, Dst, Offset, Element) {
    Eterm *untagged_source = tuple_val($Src);
    Uint size_on_heap = $Size + 1;

    /* Copy the entire tuple up-front, mimicking the behavior of the old
     * setelement/3 + set_tuple_element method. This overwrites every field
     * we're setting, but that cost is generally pretty small. */
    sys_memcpy(HTOP, untagged_source, size_on_heap * sizeof(Eterm));
    HTOP[$Offset] = $Element;

    /* We stash the contents of the destination register in SCRATCH_X_REG in
     * case it's used in subsequent `i_update_record_continue` instructions.
     * The updates have been rewritten accordingly. */
    reg[SCRATCH_X_REG] = $Dst;
    $Dst = make_tuple(HTOP);

    HTOP += size_on_heap;
}

i_update_record_in_place(Size, Src, Dst, Offset, Element) {
    Eterm *untagged_source = tuple_val($Src);
    Uint size_on_heap = $Size + 1;

    if (c_p->high_water <= untagged_source && untagged_source < HTOP) {
        /* It is safe to overwrite the old record. */
        LIGHT_SWAPOUT;

        untagged_source[$Offset] = $Element;

        /* We stash the contents of the destination register in SCRATCH_X_REG in
         * case it's used in subsequent `i_update_record_continue` instructions.
         * The updates have been rewritten accordingly. */
        reg[SCRATCH_X_REG] = $Dst;
        $Dst = $Src;

        HTOP = untagged_source + size_on_heap;
    } else {
        /* It would be unsafe to to overwrite the old record, because
         * that could cause a term on the old generation to point to
         * the young generation. */
        sys_memcpy(HTOP, untagged_source, size_on_heap * sizeof(Eterm));
        HTOP[$Offset] = $Element;

        reg[SCRATCH_X_REG] = $Dst;
        $Dst = make_tuple(HTOP);

        HTOP += size_on_heap;
        LIGHT_SWAPOUT;
    }
}

i_update_record_continue(OffsetFromEnd, Element) {
    Sint offset = -(Sint)$OffsetFromEnd;
    HTOP[offset] = $Element;
}

i_update_record_in_place_done() {
    LIGHT_SWAPIN;
}

set_tuple_element(Element, Tuple, Offset) {
    Eterm* p;

    ASSERT(is_tuple($Tuple));
    p = (Eterm *) ((unsigned char *) tuple_val($Tuple) + $Offset);
    *p = $Element;
}

swap(R1, R2) {
    Eterm V = $R1;
    $R1 = $R2;
    $R2 = V;
}

swap2(R1, R2, R3) {
    Eterm V = $R2;
    $R2 = $R1;
    $R1 = $R3;
    $R3 = V;
}

test_heap(Nh, Live) {
    $GC_TEST(0, $Nh, $Live);
}

test_heap_1_put_list(Nh, Reg) {
    $test_heap($Nh, 1);
    $put_list($Reg, x(0), x(0));
}

is_integer_allocate(Fail, Src, NeedStack, Live) {
    //| -no_prefetch
    $is_integer($Fail, $Src);
    $AH($NeedStack, 0, $Live);
}

is_nonempty_list(Fail, Src) {
    //| -no_prefetch
    if (is_not_list($Src)) {
        $FAIL($Fail);
    }
}

is_nonempty_list_allocate(Fail, Src, Need, Live) {
    //| -no_prefetch
    $is_nonempty_list($Fail, $Src);
    $AH($Need, 0, $Live);
}

is_nonempty_list_get_list(Fail, Src, Hd, Tl) {
    //| -no_prefetch
    $is_nonempty_list($Fail, $Src);
    $get_list($Src, $Hd, $Tl);
}

is_nonempty_list_get_hd(Fail, Src, Hd) {
    //| -no_prefetch
    $is_nonempty_list($Fail, $Src);
    $get_hd($Src, $Hd);
}

is_nonempty_list_get_tl(Fail, Src, Tl) {
    //| -no_prefetch
    $is_nonempty_list($Fail, $Src);
    $get_tl($Src, $Tl);
}

jump(Fail) {
    $JUMP($Fail);
}

move_jump(Lbl, Src, Dst) {
    Eterm lbl = $Lbl;
    Eterm src = $Src;
    $Dst = src;
    $JUMP(lbl);
}

//
// Test instructions.
//

is_atom(Fail, Src) {
    if (is_not_atom($Src)) {
        $FAIL($Fail);
    }
}

is_boolean(Fail, Src) {
    if (($Src) != am_true && ($Src) != am_false) {
        $FAIL($Fail);
    }
}

is_binary(Fail, Src) {
    if (is_not_bitstring($Src) || TAIL_BITS(bitstring_size($Src)) != 0) {
        $FAIL($Fail);
    }
}

is_bitstring(Fail, Src) {
  if (is_not_bitstring($Src)) {
        $FAIL($Fail);
    }
}

is_float(Fail, Src) {
    if (is_not_float($Src)) {
        $FAIL($Fail);
    }
}

is_function(Fail, Src) {
    if ( !(is_any_fun($Src)) ) {
        $FAIL($Fail);
    }
}

cold_is_function2(Fail, Fun, Arity) {
    if (erl_is_function(c_p, $Fun, $Arity) != am_true ) {
        $FAIL($Fail);
    }
}

hot_is_function2(Fail, Fun, Arity) {
    if (!is_function2($Fun, $Arity)) {
        $FAIL($Fail);
    }
}

is_integer(Fail, Src) {
    if (is_not_integer($Src)) {
        $FAIL($Fail);
    }
}

is_list(Fail, Src) {
    if (is_not_list($Src) && is_not_nil($Src)) {
        $FAIL($Fail);
    }
}

is_map(Fail, Src) {
    if (is_not_map($Src)) {
        $FAIL($Fail);
    }
}

is_nil(Fail, Src) {
    if (is_not_nil($Src)) {
        $FAIL($Fail);
    }
}

is_number(Fail, Src) {
    if (is_not_integer($Src) && is_not_float($Src)) {
        $FAIL($Fail);
    }
}

is_pid(Fail, Src) {
    if (is_not_pid($Src)) {
        $FAIL($Fail);
    }
}

is_port(Fail, Src) {
    if (is_not_port($Src)) {
        $FAIL($Fail);
    }
}

is_reference(Fail, Src) {
    if (is_not_ref($Src)) {
        $FAIL($Fail);
    }
}

is_tagged_tuple(Fail, Src, Arityval, Tag) {
    Eterm term = $Src;
    if (!(BEAM_IS_TUPLE(term) &&
          (tuple_val(term))[0] == $Arityval &&
          (tuple_val(term))[1] == $Tag)) {
        $FAIL($Fail);
    }
}

is_tagged_tuple_ff(NotTupleFail, WrongRecordFail, Src, Arityval, Tag) {
    Eterm term = $Src;
    if (is_not_tuple(term)) {
        $FAIL($NotTupleFail);
    } else if (ERTS_UNLIKELY((tuple_val(term))[0] != $Arityval ||
                             (tuple_val(term))[1] != $Tag)) {
        $FAIL($WrongRecordFail);
    }
}

is_tuple(Fail, Src) {
    if (is_not_tuple($Src)) {
        $FAIL($Fail);
    }
}

is_tuple_of_arity(Fail, Src, Arityval) {
    Eterm term = $Src;
    if (!(BEAM_IS_TUPLE(term) && *tuple_val(term) == $Arityval)) {
        $FAIL($Fail);
    }
}

test_arity(Fail, Pointer, Arity) {
    if (*tuple_val($Pointer) != $Arity) {
        $FAIL($Fail);
    }
}

test_arity_get_tuple_element(Fail, Pointer, Arity, Pos, Dst) {
    Eterm* ptr = tuple_val($Pointer);
    Eterm* src;
    if (*ptr != $Arity) {
        $FAIL($Fail);
    }
    src = ADD_BYTE_OFFSET(ptr, $Pos);
    $Dst = *src;
}

i_is_eq_exact_immed(Fail, X, Y) {
    if ($X != $Y) {
        $FAIL($Fail);
    }
}

i_is_ne_exact_immed(Fail, X, Y) {
    if ($X == $Y) {
        $FAIL($Fail);
    }
}

is_eq_exact(Fail, X, Y) {
    if (!EQ($X, $Y)) {
        $FAIL($Fail);
    }
}

i_is_eq_exact_literal(Fail, Src, Literal) {
    Eterm src = $Src;
    if (is_immed(src) || !eq(src, $Literal)) {
        $FAIL($Fail);
    }
}

is_ne_exact(Fail, X, Y) {
    if (EQ($X, $Y)) {
        $FAIL($Fail);
    }
}

i_is_ne_exact_literal(Fail, Src, Literal) {
    Eterm src = $Src;
    if (!is_immed(src) && eq(src, $Literal)) {
        $FAIL($Fail);
    }
}

is_eq(Fail, A, B) {
    Eterm a = $A;
    Eterm b = $B;
    CMP_EQ_ACTION(a, b, $FAIL($Fail));
}

is_ne(Fail, A, B) {
    Eterm a = $A;
    Eterm b = $B;
    CMP_NE_ACTION(a, b, $FAIL($Fail));
}

is_lt(Fail, X, Y) {
    CMP_LT_ACTION($X, $Y, $FAIL($Fail));
}

is_ge(Fail, X, Y) {
    CMP_GE_ACTION($X, $Y, $FAIL($Fail));
}

is_lt_literal(Fail, X, Y) {
    CMP_LT_LITERAL_ACTION($X, $Y, $FAIL($Fail));
}

is_ge_literal(Fail, X, Y) {
    CMP_GE_LITERAL_ACTION($X, $Y, $FAIL($Fail));
}

badmatch(Src) {
    c_p->fvalue = $Src;
    c_p->freason = BADMATCH;
    goto find_func_info;
    //| -no_next;
}

case_end(Src) {
    c_p->fvalue = $Src;
    c_p->freason = EXC_CASE_CLAUSE;
    goto find_func_info;
    //| -no_next;
}

if_end() {
    c_p->freason = EXC_IF_CLAUSE;
    goto find_func_info;
    //| -no_next;
}

badrecord(Src) {
    c_p->fvalue = $Src;
    c_p->freason = EXC_BADRECORD;
    goto find_func_info;
    //| -no_next;
}

system_limit_body() {
    c_p->freason = SYSTEM_LIMIT;
    $FAIL_BODY();
    //| -no_next;
}

catch(Y, Fail) {
    c_p->catches++;
    $Y = $Fail;
}

catch_end(Y) {
    /*
     * At entry:
     *
     *    x0 = Result of expression or THE_NON_VALUE
     *
     * If x0 is THE_NON_VALUE, the following registers are also set:
     *
     *    x1 = Error reason/thrown value
     *    x2 = Stacktrace
     *    x3 = Exception class
     */
    $try_end($Y);
    if (is_non_value(x(0))) {
        ASSERT(c_p->fvalue == NIL);
        ASSERT(c_p->ftrace == NIL);
        if (x(3) == am_throw) {
            x(0) = x(1);
        } else {
            if (x(3) == am_error) {
                SWAPOUT;
                x(1) = add_stacktrace(c_p, x(1), x(2));
                SWAPIN;
            }
            /* only x(1) is included in the rootset here */
            if ((E - HTOP) < (3 + S_RESERVED)) {
                $GC_SWAPOUT();
                PROCESS_MAIN_CHK_LOCKS(c_p);
                FCALLS -= erts_garbage_collect_nobump(c_p, 3, reg+1, 1, FCALLS);
                ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
                PROCESS_MAIN_CHK_LOCKS(c_p);
                SWAPIN;
                $MAYBE_EXIT_AFTER_GC();
            }
            x(0) = TUPLE2(HTOP, am_EXIT, x(1));
            HTOP += 3;
        }
    }
    CHECK_TERM(x(0));
}

try_end(Y) {
    c_p->catches--;
    make_blank($Y);
}

try_case(Y) {
    $try_end($Y);
    ASSERT(is_non_value(x(0)));
    ASSERT(c_p->fvalue == NIL);
    ASSERT(c_p->ftrace == NIL);
    x(0) = x(3);
}

try_case_end(Src) {
    c_p->fvalue = $Src;
    c_p->freason = EXC_TRY_CLAUSE;
    goto find_func_info;
    //| -no_next;
}

i_raise() {
    Eterm raise_trace = x(2);
    Eterm raise_value = x(1);

    c_p->fvalue = raise_value;
    c_p->ftrace = raise_trace;
    erts_sanitize_freason(c_p, raise_trace);
    goto find_func_info;
    //| -no_next
}

build_stacktrace() {
    SWAPOUT;
    x(0) = build_stacktrace(c_p, x(0));
    SWAPIN;
}

raw_raise() {
    //| -no_prefetch
    Eterm class = x(0);
    Eterm value = x(1);
    Eterm stacktrace = x(2);
    if (raw_raise(stacktrace, class, value, c_p)) {
        x(0) = am_badarg;
    } else {
        goto find_func_info;
    }
}

i_disabled_line_breakpoint(_Live) {
    /* NOP while the breakpoint is disabled */
}

i_enabled_line_breakpoint(Live) {
    const Export *ep;

    /* prepare return address for i_line_breakpoint_cleanup */
    $AH(0, 0, $Live);
    $SAVE_CONTINUATION_POINTER($NEXT_INSTRUCTION);

    /* prepare frame to save xregs */
    $AH($Live, 0, $Live);

    HEAVY_SWAPOUT;
    ep = erts_line_breakpoint_hit__prepare_call(c_p, I, $Live, reg, E+1);
    HEAVY_SWAPIN;

    if (ep == NULL) {
        /* We are not calling an Erlang function to handle the breakpoint,
           so we need to deallocate and continue */
        E += $Live + 2;
        $NEXT0();
    } else {
        $SAVE_CONTINUATION_POINTER((BeamInstr *) beam_i_line_breakpoint_cleanup);
        $NEXT(ep->dispatch.addresses[erts_active_code_ix()]);
    }
}

i_line_breakpoint_cleanup() {
    int live_xregs;

    HEAVY_SWAPOUT;
    live_xregs = erts_line_breakpoint_hit__cleanup(reg, E+1);
    HEAVY_SWAPIN;

    /* deallocate saved xregs and return-address that brought us here */
    E += live_xregs + 1; // AH added one more slot for the CP

    $RETURN();

    /* remove return address to next instruction and dispatch */
    E += 1;
    Goto(*I);
}

// Psuedo-instruction for signalling lambda load errors. Never actually runs.
i_lambda_error(Dummy) {

}

padding() {
}
